#!/usr/bin/perl 
use strict;
use warnings;
use bytes;
use XML::Parser;
use File::Path;
use File::Spec::Functions;

our %params = (
    working_dir         => '/home/xuqy/dmoz/',
    content_rdf_file    => 'data/content.clean',
    url_list_file       => 'list',
);

$| =1;

chdir $params{working_dir} or die "$!";

if (-d 'Top') {
    rmtree('Top', 0, 0) or die "$!";
}

my $p1 = new XML::Parser(
              Style => 'Stream',
              ErrorContext => 2,
		      ProtocolEncoding => 'UTF-8',
		      NoExpand => 1,
		      ParseParamEnt => 0);

my @pendingTag = ();
my ($topic, $url, $records, $count);

$p1->parsefile($params{content_rdf_file});

sub StartDocument {
  print  "\nLoading content records\n";
  return;
}

sub StartTag {
  m/^<(\S+)[\s>].*/;
  my $tag = $1;
  unshift @pendingTag, $tag;
  if ($tag eq 'ExternalPage') {
      $url = $_{about};
  }
  elsif($tag eq 'RDF' || $tag eq 'Topic' || $pendingTag[1] eq 'Topic' || $pendingTag[1] eq 'ExternalPage') { return; }
  else { print "Warning: Unhandled Tag [$tag] (Pending [$pendingTag[1]])\n"; }
}

sub EndTag {
  m/^<\/(\S+)>$/;
  my $tag = $1;
  shift @pendingTag;
  if($tag eq 'ExternalPage' and $url) {
    unless (&ignorable($topic)) {
      unless (-d $topic) {
        mkpath($topic, 0, 0777) or die "$!"; 
      }
      my $list_file = catfile($topic, $params{url_list_file});
      open LIST, ">> $list_file" or die "$!";
      print LIST $url, "\n";
      close LIST;
      $records++;
      unless($records % 1000) { print "Record: $records\n"; }
    }
    $count++;
  }
}

sub Text {
  if($pendingTag[0] eq 'topic') { $topic = $_; }
}

sub PI {
  return;
}

sub EndDocument {
  print "$records out of $count records loaded\n";
  return;
}

sub ignorable {
    my $topic = shift;
    $topic =~ m{^Top/(?:Adult|Kids_and_Teens|World|Regional)};
}
